{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats as stats\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston_houseprice_data = pd.read_csv(\"data/housing.data\",header=0,index_col=None,sep='\\s+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31     0  0.538  6.575  65.2  4.0900    1  296.0   \n",
       "1  0.02731   0.0   7.07     0  0.469  6.421  78.9  4.9671    2  242.0   \n",
       "2  0.02729   0.0   7.07     0  0.469  7.185  61.1  4.9671    2  242.0   \n",
       "3  0.03237   0.0   2.18     0  0.458  6.998  45.8  6.0622    3  222.0   \n",
       "4  0.06905   0.0   2.18     0  0.458  7.147  54.2  6.0622    3  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  MEDV  \n",
       "0     15.3  396.90   4.98  24.0  \n",
       "1     17.8  396.90   9.14  21.6  \n",
       "2     17.8  392.83   4.03  34.7  \n",
       "3     18.7  394.63   2.94  33.4  \n",
       "4     18.7  396.90   5.33  36.2  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston_houseprice_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sample = boston_houseprice_data.iloc[:, :-1].values\n",
    "data_label = boston_houseprice_data.iloc[:, -1].values\n",
    "bias = np.ones((data_sample.shape[0], 1))\n",
    "data_sample = np.concatenate((data_sample,bias), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean =data_sample.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.61352356e+00, 1.13636364e+01, 1.11367787e+01, 6.91699605e-02,\n",
       "       5.54695059e-01, 6.28463439e+00, 6.85749012e+01, 3.79504269e+00,\n",
       "       9.54940711e+00, 4.08237154e+02, 1.84555336e+01, 3.56674032e+02,\n",
       "       1.26530632e+01, 1.00000000e+00])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "std = data_sample.std(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\envs\\python35\\lib\\site-packages\\ipykernel_launcher.py:1: RuntimeWarning: invalid value encountered in true_divide\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-0.41978194,  0.28482986, -1.2879095 , ...,  0.44105193,\n",
       "        -1.0755623 ,         nan],\n",
       "       [-0.41733926, -0.48772236, -0.59338101, ...,  0.44105193,\n",
       "        -0.49243937,         nan],\n",
       "       [-0.41734159, -0.48772236, -0.59338101, ...,  0.39642699,\n",
       "        -1.2087274 ,         nan],\n",
       "       ...,\n",
       "       [-0.41344658, -0.48772236,  0.11573841, ...,  0.44105193,\n",
       "        -0.98304761,         nan],\n",
       "       [-0.40776407, -0.48772236,  0.11573841, ...,  0.4032249 ,\n",
       "        -0.86530163,         nan],\n",
       "       [-0.41500016, -0.48772236,  0.11573841, ...,  0.44105193,\n",
       "        -0.66905833,         nan]])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data_sample-mean)/std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "506"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6.3200e-03, 1.8000e+01, 2.3100e+00, ..., 3.9690e+02, 4.9800e+00,\n",
       "        1.0000e+00],\n",
       "       [2.7310e-02, 0.0000e+00, 7.0700e+00, ..., 3.9690e+02, 9.1400e+00,\n",
       "        1.0000e+00],\n",
       "       [2.7290e-02, 0.0000e+00, 7.0700e+00, ..., 3.9283e+02, 4.0300e+00,\n",
       "        1.0000e+00],\n",
       "       ...,\n",
       "       [6.0760e-02, 0.0000e+00, 1.1930e+01, ..., 3.9690e+02, 5.6400e+00,\n",
       "        1.0000e+00],\n",
       "       [1.0959e-01, 0.0000e+00, 1.1930e+01, ..., 3.9345e+02, 6.4800e+00,\n",
       "        1.0000e+00],\n",
       "       [4.7410e-02, 0.0000e+00, 1.1930e+01, ..., 3.9690e+02, 7.8800e+00,\n",
       "        1.0000e+00]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston_houseprice_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_length = data_label.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_length = int(data_length*0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sample_train,data_sample_test = data_sample[:train_data_length], data_sample[train_data_length:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_label_train, data_label_test = data_label[:train_data_length], data_label[train_data_length:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404, 13)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_iterations = 1000\n",
    "lr = 0.01\n",
    "weight_decay = 0.001\n",
    "train_batch_size = 16\n",
    "test_batch_size = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xavier(num_neuron_inputs, num_neuron_outputs):\n",
    "    temp1 =  np.sqrt(6) / np.sqrt(num_neuron_inputs+ num_neuron_outputs + 1)\n",
    "    weights = stats.uniform.rvs(-temp1, 2 * temp1, (num_neuron_inputs, num_neuron_outputs))\n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 503"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(inputs, weights):\n",
    "    outputs = inputs.dot(weights)\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = data_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_input_to_hidden = xavier(14,16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs_input_to_hidden = forward(inputs, weights_input_to_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  12.83131751, -100.7952357 , -108.446928  , ...,  -35.02088655,\n",
       "          75.19296777,   -1.04916604],\n",
       "       [  25.37854674,  -95.80084884, -120.18216868, ...,  -48.5360009 ,\n",
       "          50.30858591,   14.1506134 ],\n",
       "       [  28.69075407,  -99.95275788, -112.46967813, ...,  -40.30463334,\n",
       "          47.15274312,   10.61881358],\n",
       "       ...,\n",
       "       [  15.20118079,  -94.28901388, -122.29905734, ...,  -48.89831461,\n",
       "          59.55641024,    2.2523299 ],\n",
       "       [  14.23263443,  -93.45917323, -120.63612914, ...,  -47.88426205,\n",
       "          60.18498872,    0.54069848],\n",
       "       [  17.01015503,  -95.58455139, -118.27478149, ...,  -44.88523424,\n",
       "          59.40439202,   -1.42955128]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs_input_to_hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_hidden_to_output = sigmoid(outputs_input_to_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.99997324e-01, 1.67952059e-44, 7.98174025e-48, ...,\n",
       "        6.17479041e-16, 1.00000000e+00, 2.59385276e-01],\n",
       "       [1.00000000e+00, 2.47867728e-42, 6.39068361e-53, ...,\n",
       "        8.33839828e-22, 1.00000000e+00, 9.99999285e-01],\n",
       "       [1.00000000e+00, 3.90003766e-44, 1.42902362e-49, ...,\n",
       "        3.13270965e-18, 1.00000000e+00, 9.99975549e-01],\n",
       "       ...,\n",
       "       [9.99999750e-01, 1.12409125e-41, 7.69474541e-54, ...,\n",
       "        5.80405863e-22, 1.00000000e+00, 9.04851318e-01],\n",
       "       [9.99999341e-01, 2.57748878e-41, 4.05877330e-53, ...,\n",
       "        1.60003404e-21, 1.00000000e+00, 6.31974888e-01],\n",
       "       [9.99999959e-01, 3.07720745e-42, 4.30442405e-52, ...,\n",
       "        3.21063141e-20, 1.00000000e+00, 1.93168609e-01]])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_hidden_to_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 16)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_hidden_to_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_hidden_to_output = np.ones((output_hidden_to_output.shape[0], 1))\n",
    "ac1_output = np.concatenate((output_hidden_to_output, bias), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_hidden_to_output = xavier(17,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_hidden_to_output = forward(ac1_output, weights_hidden_to_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity(x):\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict = identity(output_hidden_to_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.72044781],\n",
       "       [-1.09056908],\n",
       "       [-1.09055722],\n",
       "       [-1.09056942],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-0.59119304],\n",
       "       [-0.69232122],\n",
       "       [-0.71446935],\n",
       "       [-0.70364278],\n",
       "       [-0.70785129],\n",
       "       [-0.60155034],\n",
       "       [-0.590977  ],\n",
       "       [-0.59090752],\n",
       "       [-0.72143832],\n",
       "       [-0.59086866],\n",
       "       [-0.59091133],\n",
       "       [-0.68698545],\n",
       "       [-0.86753201],\n",
       "       [-0.59395974],\n",
       "       [-0.77199189],\n",
       "       [-0.67214586],\n",
       "       [-0.67940502],\n",
       "       [-0.71582102],\n",
       "       [-0.6962535 ],\n",
       "       [-0.77777907],\n",
       "       [-0.75865435],\n",
       "       [-0.77777792],\n",
       "       [-0.71501039],\n",
       "       [-0.72650794],\n",
       "       [-0.77767836],\n",
       "       [-0.76404001],\n",
       "       [-0.78837797],\n",
       "       [-0.77764645],\n",
       "       [-0.7802912 ],\n",
       "       [-0.6003014 ],\n",
       "       [-0.5908312 ],\n",
       "       [-0.59083484],\n",
       "       [-0.59082005],\n",
       "       [-1.18519863],\n",
       "       [-1.1852019 ],\n",
       "       [-0.62990964],\n",
       "       [-0.62495708],\n",
       "       [-0.80890205],\n",
       "       [-1.089476  ],\n",
       "       [-1.08949179],\n",
       "       [-1.08566494],\n",
       "       [-1.09056931],\n",
       "       [-1.09056913],\n",
       "       [-1.09056249],\n",
       "       [-1.09056866],\n",
       "       [-1.09056943],\n",
       "       [-1.0905591 ],\n",
       "       [-1.09052807],\n",
       "       [-0.87065773],\n",
       "       [-1.08061203],\n",
       "       [-1.18520166],\n",
       "       [-1.18518772],\n",
       "       [-0.60136547],\n",
       "       [-0.86865862],\n",
       "       [-1.0693541 ],\n",
       "       [-1.17955116],\n",
       "       [-1.08986223],\n",
       "       [-0.73851488],\n",
       "       [-1.09056943],\n",
       "       [-0.69049224],\n",
       "       [-0.70248934],\n",
       "       [-0.68737795],\n",
       "       [-0.70347238],\n",
       "       [-0.69903536],\n",
       "       [-0.66804925],\n",
       "       [-0.59649005],\n",
       "       [-0.64703742],\n",
       "       [-0.67929698],\n",
       "       [-1.40587873],\n",
       "       [-0.8731019 ],\n",
       "       [-0.77971288],\n",
       "       [-0.87256586],\n",
       "       [-0.86300205],\n",
       "       [-0.87720424],\n",
       "       [-0.70442548],\n",
       "       [-1.09015781],\n",
       "       [-0.64061609],\n",
       "       [-0.77773147],\n",
       "       [-1.08679746],\n",
       "       [-1.09005763],\n",
       "       [-1.05548547],\n",
       "       [-1.09007735],\n",
       "       [-1.09003075],\n",
       "       [-0.91898271],\n",
       "       [-0.72740591],\n",
       "       [-1.00615193],\n",
       "       [-1.08671436],\n",
       "       [-0.78583539],\n",
       "       [-1.09055505],\n",
       "       [-0.59098598],\n",
       "       [-0.60421396],\n",
       "       [-1.05510711],\n",
       "       [-0.59096744],\n",
       "       [-0.66306052],\n",
       "       [-0.77777942],\n",
       "       [-0.77785728],\n",
       "       [-1.16440202],\n",
       "       [-0.77777759],\n",
       "       [-0.77777758],\n",
       "       [-0.77777756],\n",
       "       [-0.77777757],\n",
       "       [-0.7777778 ],\n",
       "       [-0.77777755],\n",
       "       [-0.77777757],\n",
       "       [-0.79418686],\n",
       "       [-0.83287279],\n",
       "       [-0.77816579],\n",
       "       [-0.77786536],\n",
       "       [-0.83329723],\n",
       "       [-0.87937901],\n",
       "       [-0.87032806],\n",
       "       [-0.82835282],\n",
       "       [-0.8845217 ],\n",
       "       [-0.87295662],\n",
       "       [-1.08953895],\n",
       "       [-1.09053557],\n",
       "       [-1.09054284],\n",
       "       [-1.09054964],\n",
       "       [-1.09054639],\n",
       "       [-1.09049058],\n",
       "       [-1.09056254],\n",
       "       [-0.77778658],\n",
       "       [-0.77777899],\n",
       "       [-0.77778324],\n",
       "       [-0.77777976],\n",
       "       [-0.77778065],\n",
       "       [-0.77779395],\n",
       "       [-0.77779235],\n",
       "       [-0.68814681],\n",
       "       [-0.77778061],\n",
       "       [-0.77799866],\n",
       "       [-0.77778014],\n",
       "       [-0.77780177],\n",
       "       [-0.77778066],\n",
       "       [-0.77812828],\n",
       "       [-0.778684  ],\n",
       "       [-0.77777757],\n",
       "       [-0.77777757],\n",
       "       [-0.7777776 ],\n",
       "       [-0.6704187 ],\n",
       "       [-0.72200985],\n",
       "       [-0.77777777],\n",
       "       [-0.77814386],\n",
       "       [-0.77784452],\n",
       "       [-0.77777831],\n",
       "       [-0.77778202],\n",
       "       [-0.78144459],\n",
       "       [-0.66733135],\n",
       "       [-0.78220778],\n",
       "       [-1.16440354],\n",
       "       [-1.16438655],\n",
       "       [-0.77780651],\n",
       "       [-0.77777151],\n",
       "       [-0.77777477],\n",
       "       [-0.783847  ],\n",
       "       [-0.77803596],\n",
       "       [-0.77777831],\n",
       "       [-0.7777834 ],\n",
       "       [-0.77777837],\n",
       "       [-0.63366324],\n",
       "       [-0.77780748],\n",
       "       [-0.89452168],\n",
       "       [-0.82268142],\n",
       "       [-0.77932135],\n",
       "       [-0.83044879],\n",
       "       [-0.77781897],\n",
       "       [-0.59235078],\n",
       "       [-0.59295775],\n",
       "       [-0.59084752],\n",
       "       [-0.59085228],\n",
       "       [-0.5908201 ],\n",
       "       [-0.59119959],\n",
       "       [-0.59102029],\n",
       "       [-1.09056939],\n",
       "       [-1.09056943],\n",
       "       [-1.09056936],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-1.09056942],\n",
       "       [-1.09056941],\n",
       "       [-0.80797467],\n",
       "       [-0.82688645],\n",
       "       [-0.80797281],\n",
       "       [-1.06962661],\n",
       "       [-0.81025373],\n",
       "       [-0.82259431],\n",
       "       [-1.18520308],\n",
       "       [-1.18520025],\n",
       "       [-1.18520235],\n",
       "       [-1.18130503],\n",
       "       [-0.80775039],\n",
       "       [-1.17374767],\n",
       "       [-0.80905205],\n",
       "       [-0.81812777],\n",
       "       [-0.72662922],\n",
       "       [-0.68580015],\n",
       "       [-1.00735458],\n",
       "       [-1.01515468],\n",
       "       [-0.59081994],\n",
       "       [-0.59085369],\n",
       "       [-0.59090906],\n",
       "       [-0.59082241],\n",
       "       [-0.68169534],\n",
       "       [-0.64357059],\n",
       "       [-0.59369722],\n",
       "       [-0.59082249],\n",
       "       [-0.59081982],\n",
       "       [-0.59128995],\n",
       "       [-0.59082276],\n",
       "       [-0.59084896],\n",
       "       [-0.73255327],\n",
       "       [-0.76469007],\n",
       "       [-0.95798224],\n",
       "       [-0.66481448],\n",
       "       [-0.70038129],\n",
       "       [-0.62002336],\n",
       "       [-0.59429403],\n",
       "       [-0.62719895],\n",
       "       [-0.68902142],\n",
       "       [-0.65263561],\n",
       "       [-0.71236073],\n",
       "       [-0.68259385],\n",
       "       [-0.66591879],\n",
       "       [-0.66672328],\n",
       "       [-0.6997238 ],\n",
       "       [-0.60452486],\n",
       "       [-0.64142001],\n",
       "       [-0.72319144],\n",
       "       [-0.63361039],\n",
       "       [-0.63242613],\n",
       "       [-0.5946036 ],\n",
       "       [-0.68544889],\n",
       "       [-0.62453701],\n",
       "       [-0.59915348],\n",
       "       [-0.71998977],\n",
       "       [-0.59723404],\n",
       "       [-0.68556133],\n",
       "       [-0.71502601],\n",
       "       [-0.71322007],\n",
       "       [-0.69603237],\n",
       "       [-0.71380137],\n",
       "       [-0.76299829],\n",
       "       [-0.68555308],\n",
       "       [-0.68564585],\n",
       "       [-0.69863425],\n",
       "       [-0.69201835],\n",
       "       [-0.68629238],\n",
       "       [-1.18338048],\n",
       "       [-1.11990546],\n",
       "       [-1.18510383],\n",
       "       [-1.09056945],\n",
       "       [-1.09057204],\n",
       "       [-1.09056956],\n",
       "       [-1.09056939],\n",
       "       [-1.09056948],\n",
       "       [-1.09056956],\n",
       "       [-1.0905695 ],\n",
       "       [-1.09056953],\n",
       "       [-1.09055299],\n",
       "       [-1.09056799],\n",
       "       [-1.09056337],\n",
       "       [-1.09055596],\n",
       "       [-1.09056943],\n",
       "       [-1.09056941],\n",
       "       [-1.09056918],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-1.09092104],\n",
       "       [-1.09057731],\n",
       "       [-1.09057009],\n",
       "       [-1.09346559],\n",
       "       [-1.09074286],\n",
       "       [-1.09056925],\n",
       "       [-1.09056943],\n",
       "       [-1.0905694 ],\n",
       "       [-1.09056943],\n",
       "       [-0.9860521 ],\n",
       "       [-1.18520372],\n",
       "       [-1.14424135],\n",
       "       [-1.18519157],\n",
       "       [-1.18191221],\n",
       "       [-1.17388205],\n",
       "       [-0.69051011],\n",
       "       [-1.1850799 ],\n",
       "       [-1.1850774 ],\n",
       "       [-1.18502504],\n",
       "       [-0.59083531],\n",
       "       [-0.5908199 ],\n",
       "       [-0.59081996],\n",
       "       [-0.59082055],\n",
       "       [-0.59082153],\n",
       "       [-0.80804863],\n",
       "       [-0.81601854],\n",
       "       [-0.80773665],\n",
       "       [-0.68596232],\n",
       "       [-0.68578321],\n",
       "       [-0.68550983],\n",
       "       [-1.09056933],\n",
       "       [-1.0905694 ],\n",
       "       [-1.09056943],\n",
       "       [-1.09056942],\n",
       "       [-0.59102423],\n",
       "       [-0.59098933],\n",
       "       [-0.71912961],\n",
       "       [-0.59082147],\n",
       "       [-0.59389046],\n",
       "       [-0.59178362],\n",
       "       [-0.59268644],\n",
       "       [-0.59114562],\n",
       "       [-0.61051826],\n",
       "       [-0.59103852],\n",
       "       [-0.59087407],\n",
       "       [-0.59083951],\n",
       "       [-0.59082904],\n",
       "       [-0.59083682],\n",
       "       [-0.59082339],\n",
       "       [-0.59092274],\n",
       "       [-0.59082037],\n",
       "       [-0.59114256],\n",
       "       [-0.5908204 ],\n",
       "       [-0.59081992],\n",
       "       [-1.41642849],\n",
       "       [-1.41681624],\n",
       "       [-1.41577689],\n",
       "       [-0.67623774],\n",
       "       [-0.68551973],\n",
       "       [-1.0905638 ],\n",
       "       [-1.0905613 ],\n",
       "       [-1.09056404],\n",
       "       [-1.09056887],\n",
       "       [-1.09056939],\n",
       "       [-1.09056527],\n",
       "       [-1.09056874],\n",
       "       [-1.09056941],\n",
       "       [-1.0900763 ],\n",
       "       [-0.87343924],\n",
       "       [-0.80790333],\n",
       "       [-0.80782306],\n",
       "       [-0.77811089],\n",
       "       [-0.77896738],\n",
       "       [-0.74761722],\n",
       "       [-1.1852037 ],\n",
       "       [-0.7067178 ],\n",
       "       [-0.70263579],\n",
       "       [-0.81471877],\n",
       "       [-0.84637764],\n",
       "       [-0.89631172],\n",
       "       [-0.73519357],\n",
       "       [-0.77166953],\n",
       "       [-1.41682123],\n",
       "       [-1.41682443],\n",
       "       [-1.41682519],\n",
       "       [-1.4168252 ],\n",
       "       [-1.41681587],\n",
       "       [-1.41664851],\n",
       "       [-1.41680691],\n",
       "       [-1.41676437],\n",
       "       [-1.41653812],\n",
       "       [-1.4167623 ],\n",
       "       [-1.39962369],\n",
       "       [-1.16440549],\n",
       "       [-1.41643217],\n",
       "       [-1.41652319],\n",
       "       [-1.41678707],\n",
       "       [-1.41659419],\n",
       "       [-1.41656712],\n",
       "       [-1.41682487],\n",
       "       [-1.41681686],\n",
       "       [-1.41682346],\n",
       "       [-1.4168245 ],\n",
       "       [-1.41682502],\n",
       "       [-1.41681835],\n",
       "       [-1.41682311],\n",
       "       [-0.88322195],\n",
       "       [-1.41682425],\n",
       "       [-1.41682504],\n",
       "       [-1.41682508],\n",
       "       [-1.41614352],\n",
       "       [-1.4168238 ],\n",
       "       [-1.41676179],\n",
       "       [-1.4168248 ],\n",
       "       [-1.41682505],\n",
       "       [-1.41682506],\n",
       "       [-1.41682485],\n",
       "       [-1.41682517],\n",
       "       [-1.41682513],\n",
       "       [-1.41682513],\n",
       "       [-1.41682514],\n",
       "       [-1.41682437],\n",
       "       [-1.41682503],\n",
       "       [-1.41682485],\n",
       "       [-1.39957874],\n",
       "       [-1.41682508],\n",
       "       [-1.41675198],\n",
       "       [-1.41682444],\n",
       "       [-1.41682074],\n",
       "       [-1.41681512],\n",
       "       [-1.41682521],\n",
       "       [-0.88395649],\n",
       "       [-1.41682466],\n",
       "       [-1.40977164],\n",
       "       [-1.4119619 ],\n",
       "       [-1.1644043 ],\n",
       "       [-0.86356479],\n",
       "       [-0.74013462],\n",
       "       [-0.60768259],\n",
       "       [-1.16447385],\n",
       "       [-1.0512891 ],\n",
       "       [-0.60815391],\n",
       "       [-0.62838865],\n",
       "       [-1.23531142],\n",
       "       [-0.95909969],\n",
       "       [-0.95231724],\n",
       "       [-1.38218183],\n",
       "       [-1.40210967],\n",
       "       [-1.37863542],\n",
       "       [-0.61043101],\n",
       "       [-0.7714216 ],\n",
       "       [-0.60744369],\n",
       "       [-0.95096946],\n",
       "       [-0.61846606],\n",
       "       [-1.30273158],\n",
       "       [-1.2043828 ],\n",
       "       [-1.30274927],\n",
       "       [-1.30195169],\n",
       "       [-1.3025365 ],\n",
       "       [-1.23803007],\n",
       "       [-1.22337634],\n",
       "       [-1.176719  ],\n",
       "       [-0.76233554],\n",
       "       [-0.60739606],\n",
       "       [-1.27827992],\n",
       "       [-1.41682521],\n",
       "       [-1.41682482],\n",
       "       [-1.41682467],\n",
       "       [-1.41682387],\n",
       "       [-1.41682393],\n",
       "       [-1.16516517],\n",
       "       [-0.93358982],\n",
       "       [-1.39568662],\n",
       "       [-1.41682463],\n",
       "       [-1.41682493],\n",
       "       [-1.31943143],\n",
       "       [-0.61049452],\n",
       "       [-1.41657465],\n",
       "       [-1.41682464],\n",
       "       [-1.41680888],\n",
       "       [-0.61113675],\n",
       "       [-0.95319476],\n",
       "       [-0.70134198],\n",
       "       [-0.66263365],\n",
       "       [-1.25504563],\n",
       "       [-1.41682522],\n",
       "       [-1.16538298],\n",
       "       [-1.41682501],\n",
       "       [-1.41682523],\n",
       "       [-1.41682479],\n",
       "       [-1.41682524],\n",
       "       [-1.41682523],\n",
       "       [-0.94257285],\n",
       "       [-1.41565086],\n",
       "       [-1.41682524],\n",
       "       [-1.41682524],\n",
       "       [-1.41682522],\n",
       "       [-1.4168249 ],\n",
       "       [-1.41682524],\n",
       "       [-1.41682522],\n",
       "       [-1.41677055],\n",
       "       [-1.3685852 ],\n",
       "       [-1.41682507],\n",
       "       [-1.41681863],\n",
       "       [-1.41682382],\n",
       "       [-1.41682514],\n",
       "       [-1.41682524],\n",
       "       [-1.41682523],\n",
       "       [-1.41682521],\n",
       "       [-1.41682524],\n",
       "       [-1.41682524],\n",
       "       [-1.41682524],\n",
       "       [-1.41682523],\n",
       "       [-1.41682524],\n",
       "       [-1.4137692 ],\n",
       "       [-1.17511382],\n",
       "       [-1.16571294],\n",
       "       [-1.37951039],\n",
       "       [-1.41618576],\n",
       "       [-0.84605785],\n",
       "       [-0.87204345],\n",
       "       [-0.9646574 ],\n",
       "       [-0.77778828],\n",
       "       [-0.77794897],\n",
       "       [-0.7798619 ],\n",
       "       [-0.77783405],\n",
       "       [-0.77778239],\n",
       "       [-0.59183608],\n",
       "       [-0.61704606],\n",
       "       [-1.04301902],\n",
       "       [-0.9066493 ],\n",
       "       [-0.68735578]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = data_label.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[24. ],\n",
       "       [21.6],\n",
       "       [34.7],\n",
       "       [33.4],\n",
       "       [36.2],\n",
       "       [28.7],\n",
       "       [22.9],\n",
       "       [27.1],\n",
       "       [16.5],\n",
       "       [18.9],\n",
       "       [15. ],\n",
       "       [18.9],\n",
       "       [21.7],\n",
       "       [20.4],\n",
       "       [18.2],\n",
       "       [19.9],\n",
       "       [23.1],\n",
       "       [17.5],\n",
       "       [20.2],\n",
       "       [18.2],\n",
       "       [13.6],\n",
       "       [19.6],\n",
       "       [15.2],\n",
       "       [14.5],\n",
       "       [15.6],\n",
       "       [13.9],\n",
       "       [16.6],\n",
       "       [14.8],\n",
       "       [18.4],\n",
       "       [21. ],\n",
       "       [12.7],\n",
       "       [14.5],\n",
       "       [13.2],\n",
       "       [13.1],\n",
       "       [13.5],\n",
       "       [18.9],\n",
       "       [20. ],\n",
       "       [21. ],\n",
       "       [24.7],\n",
       "       [30.8],\n",
       "       [34.9],\n",
       "       [26.6],\n",
       "       [25.3],\n",
       "       [24.7],\n",
       "       [21.2],\n",
       "       [19.3],\n",
       "       [20. ],\n",
       "       [16.6],\n",
       "       [14.4],\n",
       "       [19.4],\n",
       "       [19.7],\n",
       "       [20.5],\n",
       "       [25. ],\n",
       "       [23.4],\n",
       "       [18.9],\n",
       "       [35.4],\n",
       "       [24.7],\n",
       "       [31.6],\n",
       "       [23.3],\n",
       "       [19.6],\n",
       "       [18.7],\n",
       "       [16. ],\n",
       "       [22.2],\n",
       "       [25. ],\n",
       "       [33. ],\n",
       "       [23.5],\n",
       "       [19.4],\n",
       "       [22. ],\n",
       "       [17.4],\n",
       "       [20.9],\n",
       "       [24.2],\n",
       "       [21.7],\n",
       "       [22.8],\n",
       "       [23.4],\n",
       "       [24.1],\n",
       "       [21.4],\n",
       "       [20. ],\n",
       "       [20.8],\n",
       "       [21.2],\n",
       "       [20.3],\n",
       "       [28. ],\n",
       "       [23.9],\n",
       "       [24.8],\n",
       "       [22.9],\n",
       "       [23.9],\n",
       "       [26.6],\n",
       "       [22.5],\n",
       "       [22.2],\n",
       "       [23.6],\n",
       "       [28.7],\n",
       "       [22.6],\n",
       "       [22. ],\n",
       "       [22.9],\n",
       "       [25. ],\n",
       "       [20.6],\n",
       "       [28.4],\n",
       "       [21.4],\n",
       "       [38.7],\n",
       "       [43.8],\n",
       "       [33.2],\n",
       "       [27.5],\n",
       "       [26.5],\n",
       "       [18.6],\n",
       "       [19.3],\n",
       "       [20.1],\n",
       "       [19.5],\n",
       "       [19.5],\n",
       "       [20.4],\n",
       "       [19.8],\n",
       "       [19.4],\n",
       "       [21.7],\n",
       "       [22.8],\n",
       "       [18.8],\n",
       "       [18.7],\n",
       "       [18.5],\n",
       "       [18.3],\n",
       "       [21.2],\n",
       "       [19.2],\n",
       "       [20.4],\n",
       "       [19.3],\n",
       "       [22. ],\n",
       "       [20.3],\n",
       "       [20.5],\n",
       "       [17.3],\n",
       "       [18.8],\n",
       "       [21.4],\n",
       "       [15.7],\n",
       "       [16.2],\n",
       "       [18. ],\n",
       "       [14.3],\n",
       "       [19.2],\n",
       "       [19.6],\n",
       "       [23. ],\n",
       "       [18.4],\n",
       "       [15.6],\n",
       "       [18.1],\n",
       "       [17.4],\n",
       "       [17.1],\n",
       "       [13.3],\n",
       "       [17.8],\n",
       "       [14. ],\n",
       "       [14.4],\n",
       "       [13.4],\n",
       "       [15.6],\n",
       "       [11.8],\n",
       "       [13.8],\n",
       "       [15.6],\n",
       "       [14.6],\n",
       "       [17.8],\n",
       "       [15.4],\n",
       "       [21.5],\n",
       "       [19.6],\n",
       "       [15.3],\n",
       "       [19.4],\n",
       "       [17. ],\n",
       "       [15.6],\n",
       "       [13.1],\n",
       "       [41.3],\n",
       "       [24.3],\n",
       "       [23.3],\n",
       "       [27. ],\n",
       "       [50. ],\n",
       "       [50. ],\n",
       "       [50. ],\n",
       "       [22.7],\n",
       "       [25. ],\n",
       "       [50. ],\n",
       "       [23.8],\n",
       "       [23.8],\n",
       "       [22.3],\n",
       "       [17.4],\n",
       "       [19.1],\n",
       "       [23.1],\n",
       "       [23.6],\n",
       "       [22.6],\n",
       "       [29.4],\n",
       "       [23.2],\n",
       "       [24.6],\n",
       "       [29.9],\n",
       "       [37.2],\n",
       "       [39.8],\n",
       "       [36.2],\n",
       "       [37.9],\n",
       "       [32.5],\n",
       "       [26.4],\n",
       "       [29.6],\n",
       "       [50. ],\n",
       "       [32. ],\n",
       "       [29.8],\n",
       "       [34.9],\n",
       "       [37. ],\n",
       "       [30.5],\n",
       "       [36.4],\n",
       "       [31.1],\n",
       "       [29.1],\n",
       "       [50. ],\n",
       "       [33.3],\n",
       "       [30.3],\n",
       "       [34.6],\n",
       "       [34.9],\n",
       "       [32.9],\n",
       "       [24.1],\n",
       "       [42.3],\n",
       "       [48.5],\n",
       "       [50. ],\n",
       "       [22.6],\n",
       "       [24.4],\n",
       "       [22.5],\n",
       "       [24.4],\n",
       "       [20. ],\n",
       "       [21.7],\n",
       "       [19.3],\n",
       "       [22.4],\n",
       "       [28.1],\n",
       "       [23.7],\n",
       "       [25. ],\n",
       "       [23.3],\n",
       "       [28.7],\n",
       "       [21.5],\n",
       "       [23. ],\n",
       "       [26.7],\n",
       "       [21.7],\n",
       "       [27.5],\n",
       "       [30.1],\n",
       "       [44.8],\n",
       "       [50. ],\n",
       "       [37.6],\n",
       "       [31.6],\n",
       "       [46.7],\n",
       "       [31.5],\n",
       "       [24.3],\n",
       "       [31.7],\n",
       "       [41.7],\n",
       "       [48.3],\n",
       "       [29. ],\n",
       "       [24. ],\n",
       "       [25.1],\n",
       "       [31.5],\n",
       "       [23.7],\n",
       "       [23.3],\n",
       "       [22. ],\n",
       "       [20.1],\n",
       "       [22.2],\n",
       "       [23.7],\n",
       "       [17.6],\n",
       "       [18.5],\n",
       "       [24.3],\n",
       "       [20.5],\n",
       "       [24.5],\n",
       "       [26.2],\n",
       "       [24.4],\n",
       "       [24.8],\n",
       "       [29.6],\n",
       "       [42.8],\n",
       "       [21.9],\n",
       "       [20.9],\n",
       "       [44. ],\n",
       "       [50. ],\n",
       "       [36. ],\n",
       "       [30.1],\n",
       "       [33.8],\n",
       "       [43.1],\n",
       "       [48.8],\n",
       "       [31. ],\n",
       "       [36.5],\n",
       "       [22.8],\n",
       "       [30.7],\n",
       "       [50. ],\n",
       "       [43.5],\n",
       "       [20.7],\n",
       "       [21.1],\n",
       "       [25.2],\n",
       "       [24.4],\n",
       "       [35.2],\n",
       "       [32.4],\n",
       "       [32. ],\n",
       "       [33.2],\n",
       "       [33.1],\n",
       "       [29.1],\n",
       "       [35.1],\n",
       "       [45.4],\n",
       "       [35.4],\n",
       "       [46. ],\n",
       "       [50. ],\n",
       "       [32.2],\n",
       "       [22. ],\n",
       "       [20.1],\n",
       "       [23.2],\n",
       "       [22.3],\n",
       "       [24.8],\n",
       "       [28.5],\n",
       "       [37.3],\n",
       "       [27.9],\n",
       "       [23.9],\n",
       "       [21.7],\n",
       "       [28.6],\n",
       "       [27.1],\n",
       "       [20.3],\n",
       "       [22.5],\n",
       "       [29. ],\n",
       "       [24.8],\n",
       "       [22. ],\n",
       "       [26.4],\n",
       "       [33.1],\n",
       "       [36.1],\n",
       "       [28.4],\n",
       "       [33.4],\n",
       "       [28.2],\n",
       "       [22.8],\n",
       "       [20.3],\n",
       "       [16.1],\n",
       "       [22.1],\n",
       "       [19.4],\n",
       "       [21.6],\n",
       "       [23.8],\n",
       "       [16.2],\n",
       "       [17.8],\n",
       "       [19.8],\n",
       "       [23.1],\n",
       "       [21. ],\n",
       "       [23.8],\n",
       "       [23.1],\n",
       "       [20.4],\n",
       "       [18.5],\n",
       "       [25. ],\n",
       "       [24.6],\n",
       "       [23. ],\n",
       "       [22.2],\n",
       "       [19.3],\n",
       "       [22.6],\n",
       "       [19.8],\n",
       "       [17.1],\n",
       "       [19.4],\n",
       "       [22.2],\n",
       "       [20.7],\n",
       "       [21.1],\n",
       "       [19.5],\n",
       "       [18.5],\n",
       "       [20.6],\n",
       "       [19. ],\n",
       "       [18.7],\n",
       "       [32.7],\n",
       "       [16.5],\n",
       "       [23.9],\n",
       "       [31.2],\n",
       "       [17.5],\n",
       "       [17.2],\n",
       "       [23.1],\n",
       "       [24.5],\n",
       "       [26.6],\n",
       "       [22.9],\n",
       "       [24.1],\n",
       "       [18.6],\n",
       "       [30.1],\n",
       "       [18.2],\n",
       "       [20.6],\n",
       "       [17.8],\n",
       "       [21.7],\n",
       "       [22.7],\n",
       "       [22.6],\n",
       "       [25. ],\n",
       "       [19.9],\n",
       "       [20.8],\n",
       "       [16.8],\n",
       "       [21.9],\n",
       "       [27.5],\n",
       "       [21.9],\n",
       "       [23.1],\n",
       "       [50. ],\n",
       "       [50. ],\n",
       "       [50. ],\n",
       "       [50. ],\n",
       "       [50. ],\n",
       "       [13.8],\n",
       "       [13.8],\n",
       "       [15. ],\n",
       "       [13.9],\n",
       "       [13.3],\n",
       "       [13.1],\n",
       "       [10.2],\n",
       "       [10.4],\n",
       "       [10.9],\n",
       "       [11.3],\n",
       "       [12.3],\n",
       "       [ 8.8],\n",
       "       [ 7.2],\n",
       "       [10.5],\n",
       "       [ 7.4],\n",
       "       [10.2],\n",
       "       [11.5],\n",
       "       [15.1],\n",
       "       [23.2],\n",
       "       [ 9.7],\n",
       "       [13.8],\n",
       "       [12.7],\n",
       "       [13.1],\n",
       "       [12.5],\n",
       "       [ 8.5],\n",
       "       [ 5. ],\n",
       "       [ 6.3],\n",
       "       [ 5.6],\n",
       "       [ 7.2],\n",
       "       [12.1],\n",
       "       [ 8.3],\n",
       "       [ 8.5],\n",
       "       [ 5. ],\n",
       "       [11.9],\n",
       "       [27.9],\n",
       "       [17.2],\n",
       "       [27.5],\n",
       "       [15. ],\n",
       "       [17.2],\n",
       "       [17.9],\n",
       "       [16.3],\n",
       "       [ 7. ],\n",
       "       [ 7.2],\n",
       "       [ 7.5],\n",
       "       [10.4],\n",
       "       [ 8.8],\n",
       "       [ 8.4],\n",
       "       [16.7],\n",
       "       [14.2],\n",
       "       [20.8],\n",
       "       [13.4],\n",
       "       [11.7],\n",
       "       [ 8.3],\n",
       "       [10.2],\n",
       "       [10.9],\n",
       "       [11. ],\n",
       "       [ 9.5],\n",
       "       [14.5],\n",
       "       [14.1],\n",
       "       [16.1],\n",
       "       [14.3],\n",
       "       [11.7],\n",
       "       [13.4],\n",
       "       [ 9.6],\n",
       "       [ 8.7],\n",
       "       [ 8.4],\n",
       "       [12.8],\n",
       "       [10.5],\n",
       "       [17.1],\n",
       "       [18.4],\n",
       "       [15.4],\n",
       "       [10.8],\n",
       "       [11.8],\n",
       "       [14.9],\n",
       "       [12.6],\n",
       "       [14.1],\n",
       "       [13. ],\n",
       "       [13.4],\n",
       "       [15.2],\n",
       "       [16.1],\n",
       "       [17.8],\n",
       "       [14.9],\n",
       "       [14.1],\n",
       "       [12.7],\n",
       "       [13.5],\n",
       "       [14.9],\n",
       "       [20. ],\n",
       "       [16.4],\n",
       "       [17.7],\n",
       "       [19.5],\n",
       "       [20.2],\n",
       "       [21.4],\n",
       "       [19.9],\n",
       "       [19. ],\n",
       "       [19.1],\n",
       "       [19.1],\n",
       "       [20.1],\n",
       "       [19.9],\n",
       "       [19.6],\n",
       "       [23.2],\n",
       "       [29.8],\n",
       "       [13.8],\n",
       "       [13.3],\n",
       "       [16.7],\n",
       "       [12. ],\n",
       "       [14.6],\n",
       "       [21.4],\n",
       "       [23. ],\n",
       "       [23.7],\n",
       "       [25. ],\n",
       "       [21.8],\n",
       "       [20.6],\n",
       "       [21.2],\n",
       "       [19.1],\n",
       "       [20.6],\n",
       "       [15.2],\n",
       "       [ 7. ],\n",
       "       [ 8.1],\n",
       "       [13.6],\n",
       "       [20.1],\n",
       "       [21.8],\n",
       "       [24.5],\n",
       "       [23.1],\n",
       "       [19.7],\n",
       "       [18.3],\n",
       "       [21.2],\n",
       "       [17.5],\n",
       "       [16.8],\n",
       "       [22.4],\n",
       "       [20.6],\n",
       "       [23.9],\n",
       "       [22. ],\n",
       "       [11.9]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.72044781],\n",
       "       [-1.09056908]])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([24. , 21.6])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[24.72044781],\n",
       "       [22.69056908]])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label[:2].reshape(-1,1)-predict[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[611.10053978],\n",
       "       [514.86192507]])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(label[:2].reshape(-1,1)-predict[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([611.10053978, 514.86192507])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.square(label[:2].reshape(-1,1)-predict[:2]),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity_with_mse_loss(predict, label):\n",
    "    tmp1 = np.sum(np.square(label-predict),1)\n",
    "    loss = np.mean(tmp1)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "635.7892313485684"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "identity_with_mse_loss(predict,label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def der_identity_with_mse_loss(predict, label):\n",
    "    gradient = predict-label\n",
    "    return gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_outputs = der_identity_with_mse_loss(predict,label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-24.72044781],\n",
       "       [-22.69056908],\n",
       "       [-35.79055722],\n",
       "       [-34.49056942],\n",
       "       [-37.29056943],\n",
       "       [-29.79056943],\n",
       "       [-23.49119304],\n",
       "       [-27.79232122],\n",
       "       [-17.21446935],\n",
       "       [-19.60364278],\n",
       "       [-15.70785129],\n",
       "       [-19.50155034],\n",
       "       [-22.290977  ],\n",
       "       [-20.99090752],\n",
       "       [-18.92143832],\n",
       "       [-20.49086866],\n",
       "       [-23.69091133],\n",
       "       [-18.18698545],\n",
       "       [-21.06753201],\n",
       "       [-18.79395974],\n",
       "       [-14.37199189],\n",
       "       [-20.27214586],\n",
       "       [-15.87940502],\n",
       "       [-15.21582102],\n",
       "       [-16.2962535 ],\n",
       "       [-14.67777907],\n",
       "       [-17.35865435],\n",
       "       [-15.57777792],\n",
       "       [-19.11501039],\n",
       "       [-21.72650794],\n",
       "       [-13.47767836],\n",
       "       [-15.26404001],\n",
       "       [-13.98837797],\n",
       "       [-13.87764645],\n",
       "       [-14.2802912 ],\n",
       "       [-19.5003014 ],\n",
       "       [-20.5908312 ],\n",
       "       [-21.59083484],\n",
       "       [-25.29082005],\n",
       "       [-31.98519863],\n",
       "       [-36.0852019 ],\n",
       "       [-27.22990964],\n",
       "       [-25.92495708],\n",
       "       [-25.50890205],\n",
       "       [-22.289476  ],\n",
       "       [-20.38949179],\n",
       "       [-21.08566494],\n",
       "       [-17.69056931],\n",
       "       [-15.49056913],\n",
       "       [-20.49056249],\n",
       "       [-20.79056866],\n",
       "       [-21.59056943],\n",
       "       [-26.0905591 ],\n",
       "       [-24.49052807],\n",
       "       [-19.77065773],\n",
       "       [-36.48061203],\n",
       "       [-25.88520166],\n",
       "       [-32.78518772],\n",
       "       [-23.90136547],\n",
       "       [-20.46865862],\n",
       "       [-19.7693541 ],\n",
       "       [-17.17955116],\n",
       "       [-23.28986223],\n",
       "       [-25.73851488],\n",
       "       [-34.09056943],\n",
       "       [-24.19049224],\n",
       "       [-20.10248934],\n",
       "       [-22.68737795],\n",
       "       [-18.10347238],\n",
       "       [-21.59903536],\n",
       "       [-24.86804925],\n",
       "       [-22.29649005],\n",
       "       [-23.44703742],\n",
       "       [-24.07929698],\n",
       "       [-25.50587873],\n",
       "       [-22.2731019 ],\n",
       "       [-20.77971288],\n",
       "       [-21.67256586],\n",
       "       [-22.06300205],\n",
       "       [-21.17720424],\n",
       "       [-28.70442548],\n",
       "       [-24.99015781],\n",
       "       [-25.44061609],\n",
       "       [-23.67773147],\n",
       "       [-24.98679746],\n",
       "       [-27.69005763],\n",
       "       [-23.55548547],\n",
       "       [-23.29007735],\n",
       "       [-24.69003075],\n",
       "       [-29.61898271],\n",
       "       [-23.32740591],\n",
       "       [-23.00615193],\n",
       "       [-23.98671436],\n",
       "       [-25.78583539],\n",
       "       [-21.69055505],\n",
       "       [-28.99098598],\n",
       "       [-22.00421396],\n",
       "       [-39.75510711],\n",
       "       [-44.39096744],\n",
       "       [-33.86306052],\n",
       "       [-28.27777942],\n",
       "       [-27.27785728],\n",
       "       [-19.76440202],\n",
       "       [-20.07777759],\n",
       "       [-20.87777758],\n",
       "       [-20.27777756],\n",
       "       [-20.27777757],\n",
       "       [-21.1777778 ],\n",
       "       [-20.57777755],\n",
       "       [-20.17777757],\n",
       "       [-22.49418686],\n",
       "       [-23.63287279],\n",
       "       [-19.57816579],\n",
       "       [-19.47786536],\n",
       "       [-19.33329723],\n",
       "       [-19.17937901],\n",
       "       [-22.07032806],\n",
       "       [-20.02835282],\n",
       "       [-21.2845217 ],\n",
       "       [-20.17295662],\n",
       "       [-23.08953895],\n",
       "       [-21.39053557],\n",
       "       [-21.59054284],\n",
       "       [-18.39054964],\n",
       "       [-19.89054639],\n",
       "       [-22.49049058],\n",
       "       [-16.79056254],\n",
       "       [-16.97778658],\n",
       "       [-18.77777899],\n",
       "       [-15.07778324],\n",
       "       [-19.97777976],\n",
       "       [-20.37778065],\n",
       "       [-23.77779395],\n",
       "       [-19.17779235],\n",
       "       [-16.28814681],\n",
       "       [-18.87778061],\n",
       "       [-18.17799866],\n",
       "       [-17.87778014],\n",
       "       [-14.07780177],\n",
       "       [-18.57778066],\n",
       "       [-14.77812828],\n",
       "       [-15.178684  ],\n",
       "       [-14.17777757],\n",
       "       [-16.37777757],\n",
       "       [-12.5777776 ],\n",
       "       [-14.4704187 ],\n",
       "       [-16.32200985],\n",
       "       [-15.37777777],\n",
       "       [-18.57814386],\n",
       "       [-16.17784452],\n",
       "       [-22.27777831],\n",
       "       [-20.37778202],\n",
       "       [-16.08144459],\n",
       "       [-20.06733135],\n",
       "       [-17.78220778],\n",
       "       [-16.76440354],\n",
       "       [-14.26438655],\n",
       "       [-42.07780651],\n",
       "       [-25.07777151],\n",
       "       [-24.07777477],\n",
       "       [-27.783847  ],\n",
       "       [-50.77803596],\n",
       "       [-50.77777831],\n",
       "       [-50.7777834 ],\n",
       "       [-23.47777837],\n",
       "       [-25.63366324],\n",
       "       [-50.77780748],\n",
       "       [-24.69452168],\n",
       "       [-24.62268142],\n",
       "       [-23.07932135],\n",
       "       [-18.23044879],\n",
       "       [-19.87781897],\n",
       "       [-23.69235078],\n",
       "       [-24.19295775],\n",
       "       [-23.19084752],\n",
       "       [-29.99085228],\n",
       "       [-23.7908201 ],\n",
       "       [-25.19119959],\n",
       "       [-30.49102029],\n",
       "       [-38.29056939],\n",
       "       [-40.89056943],\n",
       "       [-37.29056936],\n",
       "       [-38.99056943],\n",
       "       [-33.59056943],\n",
       "       [-27.49056943],\n",
       "       [-30.69056942],\n",
       "       [-51.09056941],\n",
       "       [-32.80797467],\n",
       "       [-30.62688645],\n",
       "       [-35.70797281],\n",
       "       [-38.06962661],\n",
       "       [-31.31025373],\n",
       "       [-37.22259431],\n",
       "       [-32.28520308],\n",
       "       [-30.28520025],\n",
       "       [-51.18520235],\n",
       "       [-34.48130503],\n",
       "       [-31.10775039],\n",
       "       [-35.77374767],\n",
       "       [-35.70905205],\n",
       "       [-33.71812777],\n",
       "       [-24.82662922],\n",
       "       [-42.98580015],\n",
       "       [-49.50735458],\n",
       "       [-51.01515468],\n",
       "       [-23.19081994],\n",
       "       [-24.99085369],\n",
       "       [-23.09090906],\n",
       "       [-24.99082241],\n",
       "       [-20.68169534],\n",
       "       [-22.34357059],\n",
       "       [-19.89369722],\n",
       "       [-22.99082249],\n",
       "       [-28.69081982],\n",
       "       [-24.29128995],\n",
       "       [-25.59082276],\n",
       "       [-23.89084896],\n",
       "       [-29.43255327],\n",
       "       [-22.26469007],\n",
       "       [-23.95798224],\n",
       "       [-27.36481448],\n",
       "       [-22.40038129],\n",
       "       [-28.12002336],\n",
       "       [-30.69429403],\n",
       "       [-45.42719895],\n",
       "       [-50.68902142],\n",
       "       [-38.25263561],\n",
       "       [-32.31236073],\n",
       "       [-47.38259385],\n",
       "       [-32.16591879],\n",
       "       [-24.96672328],\n",
       "       [-32.3997238 ],\n",
       "       [-42.30452486],\n",
       "       [-48.94142001],\n",
       "       [-29.72319144],\n",
       "       [-24.63361039],\n",
       "       [-25.73242613],\n",
       "       [-32.0946036 ],\n",
       "       [-24.38544889],\n",
       "       [-23.92453701],\n",
       "       [-22.59915348],\n",
       "       [-20.81998977],\n",
       "       [-22.79723404],\n",
       "       [-24.38556133],\n",
       "       [-18.31502601],\n",
       "       [-19.21322007],\n",
       "       [-24.99603237],\n",
       "       [-21.21380137],\n",
       "       [-25.26299829],\n",
       "       [-26.88555308],\n",
       "       [-25.08564585],\n",
       "       [-25.49863425],\n",
       "       [-30.29201835],\n",
       "       [-43.48629238],\n",
       "       [-23.08338048],\n",
       "       [-22.01990546],\n",
       "       [-45.18510383],\n",
       "       [-51.09056945],\n",
       "       [-37.09057204],\n",
       "       [-31.19056956],\n",
       "       [-34.89056939],\n",
       "       [-44.19056948],\n",
       "       [-49.89056956],\n",
       "       [-32.0905695 ],\n",
       "       [-37.59056953],\n",
       "       [-23.89055299],\n",
       "       [-31.79056799],\n",
       "       [-51.09056337],\n",
       "       [-44.59055596],\n",
       "       [-21.79056943],\n",
       "       [-22.19056941],\n",
       "       [-26.29056918],\n",
       "       [-25.49056943],\n",
       "       [-36.29056943],\n",
       "       [-33.49092104],\n",
       "       [-33.09057731],\n",
       "       [-34.29057009],\n",
       "       [-34.19346559],\n",
       "       [-30.19074286],\n",
       "       [-36.19056925],\n",
       "       [-46.49056943],\n",
       "       [-36.4905694 ],\n",
       "       [-47.09056943],\n",
       "       [-50.9860521 ],\n",
       "       [-33.38520372],\n",
       "       [-23.14424135],\n",
       "       [-21.28519157],\n",
       "       [-24.38191221],\n",
       "       [-23.47388205],\n",
       "       [-25.49051011],\n",
       "       [-29.6850799 ],\n",
       "       [-38.4850774 ],\n",
       "       [-29.08502504],\n",
       "       [-24.49083531],\n",
       "       [-22.2908199 ],\n",
       "       [-29.19081996],\n",
       "       [-27.69082055],\n",
       "       [-20.89082153],\n",
       "       [-23.30804863],\n",
       "       [-29.81601854],\n",
       "       [-25.60773665],\n",
       "       [-22.68596232],\n",
       "       [-27.08578321],\n",
       "       [-33.78550983],\n",
       "       [-37.19056933],\n",
       "       [-29.4905694 ],\n",
       "       [-34.49056943],\n",
       "       [-29.29056942],\n",
       "       [-23.39102423],\n",
       "       [-20.89098933],\n",
       "       [-16.81912961],\n",
       "       [-22.69082147],\n",
       "       [-19.99389046],\n",
       "       [-22.19178362],\n",
       "       [-24.39268644],\n",
       "       [-16.79114562],\n",
       "       [-18.41051826],\n",
       "       [-20.39103852],\n",
       "       [-23.69087407],\n",
       "       [-21.59083951],\n",
       "       [-24.39082904],\n",
       "       [-23.69083682],\n",
       "       [-20.99082339],\n",
       "       [-19.09092274],\n",
       "       [-25.59082037],\n",
       "       [-25.19114256],\n",
       "       [-23.5908204 ],\n",
       "       [-22.79081992],\n",
       "       [-20.71642849],\n",
       "       [-24.01681624],\n",
       "       [-21.21577689],\n",
       "       [-17.77623774],\n",
       "       [-20.08551973],\n",
       "       [-23.2905638 ],\n",
       "       [-21.7905613 ],\n",
       "       [-22.19056404],\n",
       "       [-20.59056887],\n",
       "       [-19.59056939],\n",
       "       [-21.69056527],\n",
       "       [-20.09056874],\n",
       "       [-19.79056941],\n",
       "       [-33.7900763 ],\n",
       "       [-17.37343924],\n",
       "       [-24.70790333],\n",
       "       [-32.00782306],\n",
       "       [-18.27811089],\n",
       "       [-17.97896738],\n",
       "       [-23.84761722],\n",
       "       [-25.6852037 ],\n",
       "       [-27.3067178 ],\n",
       "       [-23.60263579],\n",
       "       [-24.91471877],\n",
       "       [-19.44637764],\n",
       "       [-30.99631172],\n",
       "       [-18.93519357],\n",
       "       [-21.37166953],\n",
       "       [-19.21682123],\n",
       "       [-23.11682443],\n",
       "       [-24.11682519],\n",
       "       [-24.0168252 ],\n",
       "       [-26.41681587],\n",
       "       [-21.31664851],\n",
       "       [-22.21680691],\n",
       "       [-18.21676437],\n",
       "       [-23.31653812],\n",
       "       [-28.9167623 ],\n",
       "       [-23.29962369],\n",
       "       [-24.26440549],\n",
       "       [-51.41643217],\n",
       "       [-51.41652319],\n",
       "       [-51.41678707],\n",
       "       [-51.41659419],\n",
       "       [-51.41656712],\n",
       "       [-15.21682487],\n",
       "       [-15.21681686],\n",
       "       [-16.41682346],\n",
       "       [-15.3168245 ],\n",
       "       [-14.71682502],\n",
       "       [-14.51681835],\n",
       "       [-11.61682311],\n",
       "       [-11.28322195],\n",
       "       [-12.31682425],\n",
       "       [-12.71682504],\n",
       "       [-13.71682508],\n",
       "       [-10.21614352],\n",
       "       [ -8.6168238 ],\n",
       "       [-11.91676179],\n",
       "       [ -8.8168248 ],\n",
       "       [-11.61682505],\n",
       "       [-12.91682506],\n",
       "       [-16.51682485],\n",
       "       [-24.61682517],\n",
       "       [-11.11682513],\n",
       "       [-15.21682513],\n",
       "       [-14.11682514],\n",
       "       [-14.51682437],\n",
       "       [-13.91682503],\n",
       "       [ -9.91682485],\n",
       "       [ -6.39957874],\n",
       "       [ -7.71682508],\n",
       "       [ -7.01675198],\n",
       "       [ -8.61682444],\n",
       "       [-13.51682074],\n",
       "       [ -9.71681512],\n",
       "       [ -9.91682521],\n",
       "       [ -5.88395649],\n",
       "       [-13.31682466],\n",
       "       [-29.30977164],\n",
       "       [-18.6119619 ],\n",
       "       [-28.6644043 ],\n",
       "       [-15.86356479],\n",
       "       [-17.94013462],\n",
       "       [-18.50768259],\n",
       "       [-17.46447385],\n",
       "       [ -8.0512891 ],\n",
       "       [ -7.80815391],\n",
       "       [ -8.12838865],\n",
       "       [-11.63531142],\n",
       "       [ -9.75909969],\n",
       "       [ -9.35231724],\n",
       "       [-18.08218183],\n",
       "       [-15.60210967],\n",
       "       [-22.17863542],\n",
       "       [-14.01043101],\n",
       "       [-12.4714216 ],\n",
       "       [ -8.90744369],\n",
       "       [-11.15096946],\n",
       "       [-11.51846606],\n",
       "       [-12.30273158],\n",
       "       [-10.7043828 ],\n",
       "       [-15.80274927],\n",
       "       [-15.40195169],\n",
       "       [-17.4025365 ],\n",
       "       [-15.53803007],\n",
       "       [-12.92337634],\n",
       "       [-14.576719  ],\n",
       "       [-10.36233554],\n",
       "       [ -9.30739606],\n",
       "       [ -9.67827992],\n",
       "       [-14.21682521],\n",
       "       [-11.91682482],\n",
       "       [-18.51682467],\n",
       "       [-19.81682387],\n",
       "       [-16.81682393],\n",
       "       [-11.96516517],\n",
       "       [-12.73358982],\n",
       "       [-16.29568662],\n",
       "       [-14.01682463],\n",
       "       [-15.51682493],\n",
       "       [-14.31943143],\n",
       "       [-14.01049452],\n",
       "       [-16.61657465],\n",
       "       [-17.51682464],\n",
       "       [-19.21680888],\n",
       "       [-15.51113675],\n",
       "       [-15.05319476],\n",
       "       [-13.40134198],\n",
       "       [-14.16263365],\n",
       "       [-16.15504563],\n",
       "       [-21.41682522],\n",
       "       [-17.56538298],\n",
       "       [-19.11682501],\n",
       "       [-20.91682523],\n",
       "       [-21.61682479],\n",
       "       [-22.81682524],\n",
       "       [-21.31682523],\n",
       "       [-19.94257285],\n",
       "       [-20.51565086],\n",
       "       [-20.51682524],\n",
       "       [-21.51682524],\n",
       "       [-21.31682522],\n",
       "       [-21.0168249 ],\n",
       "       [-24.61682524],\n",
       "       [-31.21682522],\n",
       "       [-15.21677055],\n",
       "       [-14.6685852 ],\n",
       "       [-18.11682507],\n",
       "       [-13.41681863],\n",
       "       [-16.01682382],\n",
       "       [-22.81682514],\n",
       "       [-24.41682524],\n",
       "       [-25.11682523],\n",
       "       [-26.41682521],\n",
       "       [-23.21682524],\n",
       "       [-22.01682524],\n",
       "       [-22.61682524],\n",
       "       [-20.51682523],\n",
       "       [-22.01682524],\n",
       "       [-16.6137692 ],\n",
       "       [ -8.17511382],\n",
       "       [ -9.26571294],\n",
       "       [-14.97951039],\n",
       "       [-21.51618576],\n",
       "       [-22.64605785],\n",
       "       [-25.37204345],\n",
       "       [-24.0646574 ],\n",
       "       [-20.47778828],\n",
       "       [-19.07794897],\n",
       "       [-21.9798619 ],\n",
       "       [-18.27783405],\n",
       "       [-17.57778239],\n",
       "       [-22.99183608],\n",
       "       [-21.21704606],\n",
       "       [-24.94301902],\n",
       "       [-22.9066493 ],\n",
       "       [-12.58735578]])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_weights = np.zeros((len(grad_outputs), 16, 1))\n",
    "def backward(inputs,grad_outputs,batch_size):\n",
    "    for i in np.arange(batch_size):\n",
    "        grad_weights[i,:] = np.tile(inputs[i,:], (1, 1)).T.dot(np.tile(grad_outputs[i, :], (1, 1)))\n",
    "        \n",
    "    return grad_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.72044781],\n",
       "       [-1.09056908],\n",
       "       [-1.09055722],\n",
       "       [-1.09056942],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-0.59119304],\n",
       "       [-0.69232122],\n",
       "       [-0.71446935],\n",
       "       [-0.70364278],\n",
       "       [-0.70785129],\n",
       "       [-0.60155034],\n",
       "       [-0.590977  ],\n",
       "       [-0.59090752],\n",
       "       [-0.72143832],\n",
       "       [-0.59086866],\n",
       "       [-0.59091133],\n",
       "       [-0.68698545],\n",
       "       [-0.86753201],\n",
       "       [-0.59395974],\n",
       "       [-0.77199189],\n",
       "       [-0.67214586],\n",
       "       [-0.67940502],\n",
       "       [-0.71582102],\n",
       "       [-0.6962535 ],\n",
       "       [-0.77777907],\n",
       "       [-0.75865435],\n",
       "       [-0.77777792],\n",
       "       [-0.71501039],\n",
       "       [-0.72650794],\n",
       "       [-0.77767836],\n",
       "       [-0.76404001],\n",
       "       [-0.78837797],\n",
       "       [-0.77764645],\n",
       "       [-0.7802912 ],\n",
       "       [-0.6003014 ],\n",
       "       [-0.5908312 ],\n",
       "       [-0.59083484],\n",
       "       [-0.59082005],\n",
       "       [-1.18519863],\n",
       "       [-1.1852019 ],\n",
       "       [-0.62990964],\n",
       "       [-0.62495708],\n",
       "       [-0.80890205],\n",
       "       [-1.089476  ],\n",
       "       [-1.08949179],\n",
       "       [-1.08566494],\n",
       "       [-1.09056931],\n",
       "       [-1.09056913],\n",
       "       [-1.09056249],\n",
       "       [-1.09056866],\n",
       "       [-1.09056943],\n",
       "       [-1.0905591 ],\n",
       "       [-1.09052807],\n",
       "       [-0.87065773],\n",
       "       [-1.08061203],\n",
       "       [-1.18520166],\n",
       "       [-1.18518772],\n",
       "       [-0.60136547],\n",
       "       [-0.86865862],\n",
       "       [-1.0693541 ],\n",
       "       [-1.17955116],\n",
       "       [-1.08986223],\n",
       "       [-0.73851488],\n",
       "       [-1.09056943],\n",
       "       [-0.69049224],\n",
       "       [-0.70248934],\n",
       "       [-0.68737795],\n",
       "       [-0.70347238],\n",
       "       [-0.69903536],\n",
       "       [-0.66804925],\n",
       "       [-0.59649005],\n",
       "       [-0.64703742],\n",
       "       [-0.67929698],\n",
       "       [-1.40587873],\n",
       "       [-0.8731019 ],\n",
       "       [-0.77971288],\n",
       "       [-0.87256586],\n",
       "       [-0.86300205],\n",
       "       [-0.87720424],\n",
       "       [-0.70442548],\n",
       "       [-1.09015781],\n",
       "       [-0.64061609],\n",
       "       [-0.77773147],\n",
       "       [-1.08679746],\n",
       "       [-1.09005763],\n",
       "       [-1.05548547],\n",
       "       [-1.09007735],\n",
       "       [-1.09003075],\n",
       "       [-0.91898271],\n",
       "       [-0.72740591],\n",
       "       [-1.00615193],\n",
       "       [-1.08671436],\n",
       "       [-0.78583539],\n",
       "       [-1.09055505],\n",
       "       [-0.59098598],\n",
       "       [-0.60421396],\n",
       "       [-1.05510711],\n",
       "       [-0.59096744],\n",
       "       [-0.66306052],\n",
       "       [-0.77777942],\n",
       "       [-0.77785728],\n",
       "       [-1.16440202],\n",
       "       [-0.77777759],\n",
       "       [-0.77777758],\n",
       "       [-0.77777756],\n",
       "       [-0.77777757],\n",
       "       [-0.7777778 ],\n",
       "       [-0.77777755],\n",
       "       [-0.77777757],\n",
       "       [-0.79418686],\n",
       "       [-0.83287279],\n",
       "       [-0.77816579],\n",
       "       [-0.77786536],\n",
       "       [-0.83329723],\n",
       "       [-0.87937901],\n",
       "       [-0.87032806],\n",
       "       [-0.82835282],\n",
       "       [-0.8845217 ],\n",
       "       [-0.87295662],\n",
       "       [-1.08953895],\n",
       "       [-1.09053557],\n",
       "       [-1.09054284],\n",
       "       [-1.09054964],\n",
       "       [-1.09054639],\n",
       "       [-1.09049058],\n",
       "       [-1.09056254],\n",
       "       [-0.77778658],\n",
       "       [-0.77777899],\n",
       "       [-0.77778324],\n",
       "       [-0.77777976],\n",
       "       [-0.77778065],\n",
       "       [-0.77779395],\n",
       "       [-0.77779235],\n",
       "       [-0.68814681],\n",
       "       [-0.77778061],\n",
       "       [-0.77799866],\n",
       "       [-0.77778014],\n",
       "       [-0.77780177],\n",
       "       [-0.77778066],\n",
       "       [-0.77812828],\n",
       "       [-0.778684  ],\n",
       "       [-0.77777757],\n",
       "       [-0.77777757],\n",
       "       [-0.7777776 ],\n",
       "       [-0.6704187 ],\n",
       "       [-0.72200985],\n",
       "       [-0.77777777],\n",
       "       [-0.77814386],\n",
       "       [-0.77784452],\n",
       "       [-0.77777831],\n",
       "       [-0.77778202],\n",
       "       [-0.78144459],\n",
       "       [-0.66733135],\n",
       "       [-0.78220778],\n",
       "       [-1.16440354],\n",
       "       [-1.16438655],\n",
       "       [-0.77780651],\n",
       "       [-0.77777151],\n",
       "       [-0.77777477],\n",
       "       [-0.783847  ],\n",
       "       [-0.77803596],\n",
       "       [-0.77777831],\n",
       "       [-0.7777834 ],\n",
       "       [-0.77777837],\n",
       "       [-0.63366324],\n",
       "       [-0.77780748],\n",
       "       [-0.89452168],\n",
       "       [-0.82268142],\n",
       "       [-0.77932135],\n",
       "       [-0.83044879],\n",
       "       [-0.77781897],\n",
       "       [-0.59235078],\n",
       "       [-0.59295775],\n",
       "       [-0.59084752],\n",
       "       [-0.59085228],\n",
       "       [-0.5908201 ],\n",
       "       [-0.59119959],\n",
       "       [-0.59102029],\n",
       "       [-1.09056939],\n",
       "       [-1.09056943],\n",
       "       [-1.09056936],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-1.09056942],\n",
       "       [-1.09056941],\n",
       "       [-0.80797467],\n",
       "       [-0.82688645],\n",
       "       [-0.80797281],\n",
       "       [-1.06962661],\n",
       "       [-0.81025373],\n",
       "       [-0.82259431],\n",
       "       [-1.18520308],\n",
       "       [-1.18520025],\n",
       "       [-1.18520235],\n",
       "       [-1.18130503],\n",
       "       [-0.80775039],\n",
       "       [-1.17374767],\n",
       "       [-0.80905205],\n",
       "       [-0.81812777],\n",
       "       [-0.72662922],\n",
       "       [-0.68580015],\n",
       "       [-1.00735458],\n",
       "       [-1.01515468],\n",
       "       [-0.59081994],\n",
       "       [-0.59085369],\n",
       "       [-0.59090906],\n",
       "       [-0.59082241],\n",
       "       [-0.68169534],\n",
       "       [-0.64357059],\n",
       "       [-0.59369722],\n",
       "       [-0.59082249],\n",
       "       [-0.59081982],\n",
       "       [-0.59128995],\n",
       "       [-0.59082276],\n",
       "       [-0.59084896],\n",
       "       [-0.73255327],\n",
       "       [-0.76469007],\n",
       "       [-0.95798224],\n",
       "       [-0.66481448],\n",
       "       [-0.70038129],\n",
       "       [-0.62002336],\n",
       "       [-0.59429403],\n",
       "       [-0.62719895],\n",
       "       [-0.68902142],\n",
       "       [-0.65263561],\n",
       "       [-0.71236073],\n",
       "       [-0.68259385],\n",
       "       [-0.66591879],\n",
       "       [-0.66672328],\n",
       "       [-0.6997238 ],\n",
       "       [-0.60452486],\n",
       "       [-0.64142001],\n",
       "       [-0.72319144],\n",
       "       [-0.63361039],\n",
       "       [-0.63242613],\n",
       "       [-0.5946036 ],\n",
       "       [-0.68544889],\n",
       "       [-0.62453701],\n",
       "       [-0.59915348],\n",
       "       [-0.71998977],\n",
       "       [-0.59723404],\n",
       "       [-0.68556133],\n",
       "       [-0.71502601],\n",
       "       [-0.71322007],\n",
       "       [-0.69603237],\n",
       "       [-0.71380137],\n",
       "       [-0.76299829],\n",
       "       [-0.68555308],\n",
       "       [-0.68564585],\n",
       "       [-0.69863425],\n",
       "       [-0.69201835],\n",
       "       [-0.68629238],\n",
       "       [-1.18338048],\n",
       "       [-1.11990546],\n",
       "       [-1.18510383],\n",
       "       [-1.09056945],\n",
       "       [-1.09057204],\n",
       "       [-1.09056956],\n",
       "       [-1.09056939],\n",
       "       [-1.09056948],\n",
       "       [-1.09056956],\n",
       "       [-1.0905695 ],\n",
       "       [-1.09056953],\n",
       "       [-1.09055299],\n",
       "       [-1.09056799],\n",
       "       [-1.09056337],\n",
       "       [-1.09055596],\n",
       "       [-1.09056943],\n",
       "       [-1.09056941],\n",
       "       [-1.09056918],\n",
       "       [-1.09056943],\n",
       "       [-1.09056943],\n",
       "       [-1.09092104],\n",
       "       [-1.09057731],\n",
       "       [-1.09057009],\n",
       "       [-1.09346559],\n",
       "       [-1.09074286],\n",
       "       [-1.09056925],\n",
       "       [-1.09056943],\n",
       "       [-1.0905694 ],\n",
       "       [-1.09056943],\n",
       "       [-0.9860521 ],\n",
       "       [-1.18520372],\n",
       "       [-1.14424135],\n",
       "       [-1.18519157],\n",
       "       [-1.18191221],\n",
       "       [-1.17388205],\n",
       "       [-0.69051011],\n",
       "       [-1.1850799 ],\n",
       "       [-1.1850774 ],\n",
       "       [-1.18502504],\n",
       "       [-0.59083531],\n",
       "       [-0.5908199 ],\n",
       "       [-0.59081996],\n",
       "       [-0.59082055],\n",
       "       [-0.59082153],\n",
       "       [-0.80804863],\n",
       "       [-0.81601854],\n",
       "       [-0.80773665],\n",
       "       [-0.68596232],\n",
       "       [-0.68578321],\n",
       "       [-0.68550983],\n",
       "       [-1.09056933],\n",
       "       [-1.0905694 ],\n",
       "       [-1.09056943],\n",
       "       [-1.09056942],\n",
       "       [-0.59102423],\n",
       "       [-0.59098933],\n",
       "       [-0.71912961],\n",
       "       [-0.59082147],\n",
       "       [-0.59389046],\n",
       "       [-0.59178362],\n",
       "       [-0.59268644],\n",
       "       [-0.59114562],\n",
       "       [-0.61051826],\n",
       "       [-0.59103852],\n",
       "       [-0.59087407],\n",
       "       [-0.59083951],\n",
       "       [-0.59082904],\n",
       "       [-0.59083682],\n",
       "       [-0.59082339],\n",
       "       [-0.59092274],\n",
       "       [-0.59082037],\n",
       "       [-0.59114256],\n",
       "       [-0.5908204 ],\n",
       "       [-0.59081992],\n",
       "       [-1.41642849],\n",
       "       [-1.41681624],\n",
       "       [-1.41577689],\n",
       "       [-0.67623774],\n",
       "       [-0.68551973],\n",
       "       [-1.0905638 ],\n",
       "       [-1.0905613 ],\n",
       "       [-1.09056404],\n",
       "       [-1.09056887],\n",
       "       [-1.09056939],\n",
       "       [-1.09056527],\n",
       "       [-1.09056874],\n",
       "       [-1.09056941],\n",
       "       [-1.0900763 ],\n",
       "       [-0.87343924],\n",
       "       [-0.80790333],\n",
       "       [-0.80782306],\n",
       "       [-0.77811089],\n",
       "       [-0.77896738],\n",
       "       [-0.74761722],\n",
       "       [-1.1852037 ],\n",
       "       [-0.7067178 ],\n",
       "       [-0.70263579],\n",
       "       [-0.81471877],\n",
       "       [-0.84637764],\n",
       "       [-0.89631172],\n",
       "       [-0.73519357],\n",
       "       [-0.77166953],\n",
       "       [-1.41682123],\n",
       "       [-1.41682443],\n",
       "       [-1.41682519],\n",
       "       [-1.4168252 ],\n",
       "       [-1.41681587],\n",
       "       [-1.41664851],\n",
       "       [-1.41680691],\n",
       "       [-1.41676437],\n",
       "       [-1.41653812],\n",
       "       [-1.4167623 ],\n",
       "       [-1.39962369],\n",
       "       [-1.16440549],\n",
       "       [-1.41643217],\n",
       "       [-1.41652319],\n",
       "       [-1.41678707],\n",
       "       [-1.41659419],\n",
       "       [-1.41656712],\n",
       "       [-1.41682487],\n",
       "       [-1.41681686],\n",
       "       [-1.41682346],\n",
       "       [-1.4168245 ],\n",
       "       [-1.41682502],\n",
       "       [-1.41681835],\n",
       "       [-1.41682311],\n",
       "       [-0.88322195],\n",
       "       [-1.41682425],\n",
       "       [-1.41682504],\n",
       "       [-1.41682508],\n",
       "       [-1.41614352],\n",
       "       [-1.4168238 ],\n",
       "       [-1.41676179],\n",
       "       [-1.4168248 ],\n",
       "       [-1.41682505],\n",
       "       [-1.41682506],\n",
       "       [-1.41682485],\n",
       "       [-1.41682517],\n",
       "       [-1.41682513],\n",
       "       [-1.41682513],\n",
       "       [-1.41682514],\n",
       "       [-1.41682437],\n",
       "       [-1.41682503],\n",
       "       [-1.41682485],\n",
       "       [-1.39957874],\n",
       "       [-1.41682508],\n",
       "       [-1.41675198],\n",
       "       [-1.41682444],\n",
       "       [-1.41682074],\n",
       "       [-1.41681512],\n",
       "       [-1.41682521],\n",
       "       [-0.88395649],\n",
       "       [-1.41682466],\n",
       "       [-1.40977164],\n",
       "       [-1.4119619 ],\n",
       "       [-1.1644043 ],\n",
       "       [-0.86356479],\n",
       "       [-0.74013462],\n",
       "       [-0.60768259],\n",
       "       [-1.16447385],\n",
       "       [-1.0512891 ],\n",
       "       [-0.60815391],\n",
       "       [-0.62838865],\n",
       "       [-1.23531142],\n",
       "       [-0.95909969],\n",
       "       [-0.95231724],\n",
       "       [-1.38218183],\n",
       "       [-1.40210967],\n",
       "       [-1.37863542],\n",
       "       [-0.61043101],\n",
       "       [-0.7714216 ],\n",
       "       [-0.60744369],\n",
       "       [-0.95096946],\n",
       "       [-0.61846606],\n",
       "       [-1.30273158],\n",
       "       [-1.2043828 ],\n",
       "       [-1.30274927],\n",
       "       [-1.30195169],\n",
       "       [-1.3025365 ],\n",
       "       [-1.23803007],\n",
       "       [-1.22337634],\n",
       "       [-1.176719  ],\n",
       "       [-0.76233554],\n",
       "       [-0.60739606],\n",
       "       [-1.27827992],\n",
       "       [-1.41682521],\n",
       "       [-1.41682482],\n",
       "       [-1.41682467],\n",
       "       [-1.41682387],\n",
       "       [-1.41682393],\n",
       "       [-1.16516517],\n",
       "       [-0.93358982],\n",
       "       [-1.39568662],\n",
       "       [-1.41682463],\n",
       "       [-1.41682493],\n",
       "       [-1.31943143],\n",
       "       [-0.61049452],\n",
       "       [-1.41657465],\n",
       "       [-1.41682464],\n",
       "       [-1.41680888],\n",
       "       [-0.61113675],\n",
       "       [-0.95319476],\n",
       "       [-0.70134198],\n",
       "       [-0.66263365],\n",
       "       [-1.25504563],\n",
       "       [-1.41682522],\n",
       "       [-1.16538298],\n",
       "       [-1.41682501],\n",
       "       [-1.41682523],\n",
       "       [-1.41682479],\n",
       "       [-1.41682524],\n",
       "       [-1.41682523],\n",
       "       [-0.94257285],\n",
       "       [-1.41565086],\n",
       "       [-1.41682524],\n",
       "       [-1.41682524],\n",
       "       [-1.41682522],\n",
       "       [-1.4168249 ],\n",
       "       [-1.41682524],\n",
       "       [-1.41682522],\n",
       "       [-1.41677055],\n",
       "       [-1.3685852 ],\n",
       "       [-1.41682507],\n",
       "       [-1.41681863],\n",
       "       [-1.41682382],\n",
       "       [-1.41682514],\n",
       "       [-1.41682524],\n",
       "       [-1.41682523],\n",
       "       [-1.41682521],\n",
       "       [-1.41682524],\n",
       "       [-1.41682524],\n",
       "       [-1.41682524],\n",
       "       [-1.41682523],\n",
       "       [-1.41682524],\n",
       "       [-1.4137692 ],\n",
       "       [-1.17511382],\n",
       "       [-1.16571294],\n",
       "       [-1.37951039],\n",
       "       [-1.41618576],\n",
       "       [-0.84605785],\n",
       "       [-0.87204345],\n",
       "       [-0.9646574 ],\n",
       "       [-0.77778828],\n",
       "       [-0.77794897],\n",
       "       [-0.7798619 ],\n",
       "       [-0.77783405],\n",
       "       [-0.77778239],\n",
       "       [-0.59183608],\n",
       "       [-0.61704606],\n",
       "       [-1.04301902],\n",
       "       [-0.9066493 ],\n",
       "       [-0.68735578]])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_hidden_to_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_hidden_to_output = backward(output_hidden_to_output,grad_outputs,len(grad_outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 16, 1)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_hidden_to_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update(grad_weights,weights,update_function):\n",
    "    grad_weights_average = np.mean(grad_weights, 0)\n",
    "    weights = update_function(weights,grad_weights_average)\n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 梯度下降法\n",
    "def batch_gradient_descent(weights, grad_weights):  \n",
    "    lr = 0.001\n",
    "    weights_now = weights + lr * grad_weights\n",
    "    return weights_now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_hidden_to_output_new = update(grad_hidden_to_output,weights_hidden_to_output,batch_gradient_descent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.07070002, -0.35480579, -0.05962447,  0.16788701, -0.37981698,\n",
       "        0.25069673, -0.57178228,  0.17528706, -0.27925468,  0.24480724,\n",
       "        0.09595551,  0.23322323, -0.33232044,  0.42877296, -0.09034356,\n",
       "       -0.1885033 ])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_hidden_to_output_new.reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_hidden_output = grad_outputs.dot(weights_hidden_to_output.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -1.87047692,   8.49418743,   1.30402641,  -4.23780181,\n",
       "          9.10342132,  -6.25491878,  13.77939947,  -4.41805563,\n",
       "          6.65387901,  -6.11145987,  -2.48566161,  -5.82929128,\n",
       "          7.946479  , -10.59258064,   2.05229586,   4.44331552]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_hidden_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def der_sigmoid(x):\n",
    "    return sigmoid(x) * (1 - sigmoid(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_simoid_inputs = grad_hidden_output*der_sigmoid(output_hidden_to_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.46761923,  2.12354686,  0.3260066 , -1.05945045,  2.27585533,\n",
       "        -1.5637297 ,  3.44484987, -0.86864246,  1.30823201, -1.52786497,\n",
       "        -0.6214154 , -1.45732282,  1.98661975, -2.64814516,  0.40350586,\n",
       "         0.87360885]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_simoid_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_input_to_hidden = backward(outputs_input_to_hidden,grad_simoid_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_input_to_hidden_new = update(grad_input_to_hidden,weights_input_to_hidden,batch_gradient_descent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.30869138, -0.36365895,  0.41273301, -0.10332444, -0.42570369,\n",
       "         0.3565688 ,  0.12870381, -0.14816078, -0.14102841, -0.27617893,\n",
       "        -0.09429246,  0.43725663,  0.01155555,  0.33841246, -0.01513624,\n",
       "        -0.1320727 ],\n",
       "       [ 0.01126416, -0.05446262,  0.35645225, -0.08239109, -0.2666968 ,\n",
       "         0.53130775,  0.09750704,  0.22560708,  0.32399666,  0.19735464,\n",
       "        -0.28575477,  0.06157891,  0.21231716, -0.11248171, -0.43840381,\n",
       "        -0.2382428 ],\n",
       "       [ 0.24423992, -0.28423604, -0.41739189, -0.31646319,  0.16640596,\n",
       "         0.53469557, -0.50095401,  0.30239648, -0.01849383, -0.16899224,\n",
       "        -0.35190935, -0.09046623, -0.44549153,  0.43186524, -0.00540715,\n",
       "         0.06518796],\n",
       "       [-0.1754075 ,  0.28239922,  0.07570541,  0.09208882, -0.41453816,\n",
       "        -0.12403823, -0.55702845, -0.06040721,  0.28948081,  0.07719998,\n",
       "         0.45241469, -0.02009017, -0.05459301, -0.03366673,  0.14814532,\n",
       "         0.09463739],\n",
       "       [ 0.46563392, -0.04738364, -0.09188664,  0.38135077, -0.02022596,\n",
       "        -0.18118461,  0.23880524,  0.10853515,  0.3375276 ,  0.34254579,\n",
       "         0.17056079,  0.08461359, -0.15037556,  0.59202159, -0.15094088,\n",
       "         0.12892392],\n",
       "       [ 0.34844109,  0.03162953, -0.0882978 , -0.26666443, -0.14232395,\n",
       "         0.17142224, -0.58643196, -0.26128205,  0.12573108,  0.33664067,\n",
       "        -0.06818396,  0.31871896,  0.1237379 , -0.10597011, -0.13445259,\n",
       "        -0.03072581],\n",
       "       [-0.21713849, -0.38279894, -0.30263761,  0.10574734,  0.28815611,\n",
       "         0.43115158, -0.43083556,  0.31779161,  0.10638132,  0.30996981,\n",
       "         0.23655408,  0.3780268 , -0.12680772, -0.03377217,  0.29466431,\n",
       "         0.14797415],\n",
       "       [-0.21180183,  0.12171245,  0.05318741,  0.4421782 , -0.19482481,\n",
       "         0.14482497, -0.38215607, -0.19690912, -0.34141625, -0.3483092 ,\n",
       "        -0.28561999,  0.2553273 ,  0.13939968, -0.27662248,  0.16476697,\n",
       "        -0.01968906],\n",
       "       [ 0.14876997, -0.11483937,  0.2776837 ,  0.32289726, -0.52062189,\n",
       "         0.0716138 , -0.11796333, -0.11655989, -0.10815357,  0.32414008,\n",
       "         0.03966797, -0.03775076, -0.08374054,  0.42920271, -0.3496139 ,\n",
       "        -0.40126464],\n",
       "       [-0.30537432, -0.23406654,  0.22372538,  0.06873656, -0.28429705,\n",
       "        -0.23171161, -0.32157448,  0.3439719 , -0.34606904, -0.27487583,\n",
       "        -0.27988591, -0.12547426, -0.44741714,  0.31101564, -0.04893527,\n",
       "        -0.16664632],\n",
       "       [-0.38782211, -0.06515768,  0.15987366, -0.3452445 , -0.40447404,\n",
       "        -0.01540079, -0.00609313,  0.24748916,  0.04439294, -0.05278793,\n",
       "         0.26254867,  0.03316445,  0.29052432,  0.08471599,  0.37878375,\n",
       "         0.36298533],\n",
       "       [ 0.26205544, -0.42587747, -0.23488495, -0.05986661, -0.45353798,\n",
       "         0.06502795, -0.52722224, -0.13600324,  0.35157402, -0.263364  ,\n",
       "        -0.22406016,  0.16033792, -0.07058715, -0.25670761,  0.38258132,\n",
       "         0.1061511 ],\n",
       "       [ 0.46979032, -0.27540897, -0.45574841,  0.46404112, -0.2790613 ,\n",
       "        -0.24529278, -0.53247109,  0.03362287, -0.18589008,  0.22000025,\n",
       "         0.00646693,  0.31755893, -0.34418391,  0.48875936, -0.45090117,\n",
       "        -0.48328748]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_input_to_hidden_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmaxwithloss(inputs, label):\n",
    "    temp1 = np.exp(inputs)\n",
    "    probability = temp1 / (np.tile(np.sum(temp1, 1), (inputs.shape[1], 1))).T\n",
    "    temp3 = np.argmax(label, 1)   #纵坐标\n",
    "    temp4 = [probability[i, j] for (i, j) in zip(np.arange(label.shape[0]), temp3)]\n",
    "    loss = -1 * np.mean(np.log(temp4))\n",
    "    return loss\n",
    "def der_softmaxwithloss(inputs, label):\n",
    "    temp1 = np.exp(inputs)\n",
    "    temp2 = np.sum(temp1, 1)  #它得到的是一维的向量;\n",
    "    probability = temp1 / (np.tile(temp2, (inputs.shape[1], 1))).T\n",
    "    gradient = probability - label\n",
    "    return gradient"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:python35]",
   "language": "python",
   "name": "conda-env-python35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
